import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/DNN_3d_TFM'
TIME_STEPS=432 #3d
UNITS=97
DROPOUT=0.795
ACTIVATION='sigmoid'
OPTIMIZER='adamax'
EPOCHS=98
BATCHSIZE=43
VALIDATIONSPLIT=0.1
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (2685, 432, 1) y_train shape: (2685,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(Dense(units=UNITS, input_shape=(X_train.shape[1], X_train.shape[2]), activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dropout(rate=DROPOUT))
model.add(Dense(X_train.shape[2],activation=ACTIVATION))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=[rmse,'mse'])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense (Dense) (None, 432, 97) 194 _________________________________________________________________ dense_1 (Dense) (None, 432, 16) 1568 _________________________________________________________________ dropout (Dropout) (None, 432, 16) 0 _________________________________________________________________ dense_2 (Dense) (None, 432, 1) 17 ================================================================= Total params: 1,779 Trainable params: 1,779 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/98 57/57 [==============================] - 1s 21ms/step - loss: 0.9370 - rmse: 0.9398 - mse: 1.2110 - val_loss: 1.3294 - val_rmse: 1.3298 - val_mse: 1.8141 Epoch 2/98 57/57 [==============================] - 1s 17ms/step - loss: 0.9133 - rmse: 0.9188 - mse: 1.1645 - val_loss: 1.2641 - val_rmse: 1.2651 - val_mse: 1.6453 Epoch 3/98 57/57 [==============================] - 1s 18ms/step - loss: 0.8954 - rmse: 0.9044 - mse: 1.1306 - val_loss: 1.1958 - val_rmse: 1.1972 - val_mse: 1.4773 Epoch 4/98 57/57 [==============================] - 1s 19ms/step - loss: 0.8776 - rmse: 0.8903 - mse: 1.0996 - val_loss: 1.1330 - val_rmse: 1.1346 - val_mse: 1.3303 Epoch 5/98 57/57 [==============================] - 1s 17ms/step - loss: 0.8623 - rmse: 0.8783 - mse: 1.0750 - val_loss: 1.0749 - val_rmse: 1.0768 - val_mse: 1.2015 Epoch 6/98 57/57 [==============================] - 1s 18ms/step - loss: 0.8489 - rmse: 0.8671 - mse: 1.0547 - val_loss: 1.0291 - val_rmse: 1.0308 - val_mse: 1.1039 Epoch 7/98 57/57 [==============================] - 1s 20ms/step - loss: 0.8384 - rmse: 0.8582 - mse: 1.0402 - val_loss: 0.9970 - val_rmse: 0.9984 - val_mse: 1.0377 Epoch 8/98 57/57 [==============================] - 1s 19ms/step - loss: 0.8307 - rmse: 0.8514 - mse: 1.0300 - val_loss: 0.9747 - val_rmse: 0.9759 - val_mse: 0.9930 Epoch 9/98 57/57 [==============================] - 1s 17ms/step - loss: 0.8255 - rmse: 0.8468 - mse: 1.0246 - val_loss: 0.9594 - val_rmse: 0.9603 - val_mse: 0.9627 Epoch 10/98 57/57 [==============================] - 1s 17ms/step - loss: 0.8213 - rmse: 0.8428 - mse: 1.0193 - val_loss: 0.9487 - val_rmse: 0.9494 - val_mse: 0.9419 Epoch 11/98 57/57 [==============================] - 1s 17ms/step - loss: 0.8178 - rmse: 0.8394 - mse: 1.0156 - val_loss: 0.9411 - val_rmse: 0.9416 - val_mse: 0.9272 Epoch 12/98 57/57 [==============================] - 1s 16ms/step - loss: 0.8146 - rmse: 0.8361 - mse: 1.0117 - val_loss: 0.9355 - val_rmse: 0.9359 - val_mse: 0.9166 Epoch 13/98 57/57 [==============================] - 1s 15ms/step - loss: 0.8123 - rmse: 0.8336 - mse: 1.0092 - val_loss: 0.9315 - val_rmse: 0.9317 - val_mse: 0.9089 Epoch 14/98 57/57 [==============================] - 1s 15ms/step - loss: 0.8107 - rmse: 0.8319 - mse: 1.0079 - val_loss: 0.9284 - val_rmse: 0.9286 - val_mse: 0.9031 Epoch 15/98 57/57 [==============================] - 1s 15ms/step - loss: 0.8089 - rmse: 0.8300 - mse: 1.0063 - val_loss: 0.9261 - val_rmse: 0.9263 - val_mse: 0.8989 Epoch 16/98 57/57 [==============================] - 1s 16ms/step - loss: 0.8073 - rmse: 0.8281 - mse: 1.0043 - val_loss: 0.9244 - val_rmse: 0.9245 - val_mse: 0.8956 Epoch 17/98 57/57 [==============================] - 1s 16ms/step - loss: 0.8066 - rmse: 0.8274 - mse: 1.0044 - val_loss: 0.9231 - val_rmse: 0.9232 - val_mse: 0.8932 Epoch 18/98 57/57 [==============================] - 1s 17ms/step - loss: 0.8054 - rmse: 0.8260 - mse: 1.0026 - val_loss: 0.9221 - val_rmse: 0.9222 - val_mse: 0.8914 Epoch 19/98 57/57 [==============================] - 1s 19ms/step - loss: 0.8047 - rmse: 0.8250 - mse: 1.0024 - val_loss: 0.9213 - val_rmse: 0.9214 - val_mse: 0.8899 Epoch 20/98 57/57 [==============================] - 1s 16ms/step - loss: 0.8038 - rmse: 0.8240 - mse: 1.0014 - val_loss: 0.9207 - val_rmse: 0.9207 - val_mse: 0.8888 Epoch 21/98 57/57 [==============================] - 1s 16ms/step - loss: 0.8036 - rmse: 0.8236 - mse: 1.0016 - val_loss: 0.9202 - val_rmse: 0.9202 - val_mse: 0.8878 Epoch 22/98 57/57 [==============================] - 1s 20ms/step - loss: 0.8029 - rmse: 0.8227 - mse: 1.0007 - val_loss: 0.9198 - val_rmse: 0.9198 - val_mse: 0.8871 Epoch 23/98 57/57 [==============================] - 1s 18ms/step - loss: 0.8022 - rmse: 0.8218 - mse: 1.0000 - val_loss: 0.9194 - val_rmse: 0.9195 - val_mse: 0.8865 Epoch 24/98 57/57 [==============================] - 1s 19ms/step - loss: 0.8017 - rmse: 0.8210 - mse: 0.9994 - val_loss: 0.9192 - val_rmse: 0.9192 - val_mse: 0.8860 Epoch 25/98 57/57 [==============================] - 1s 19ms/step - loss: 0.8012 - rmse: 0.8202 - mse: 0.9987 - val_loss: 0.9189 - val_rmse: 0.9190 - val_mse: 0.8856 Epoch 26/98 57/57 [==============================] - 1s 17ms/step - loss: 0.8008 - rmse: 0.8196 - mse: 0.9981 - val_loss: 0.9187 - val_rmse: 0.9188 - val_mse: 0.8852 Epoch 27/98 57/57 [==============================] - 1s 19ms/step - loss: 0.8004 - rmse: 0.8188 - mse: 0.9980 - val_loss: 0.9186 - val_rmse: 0.9186 - val_mse: 0.8849 Epoch 28/98 57/57 [==============================] - 1s 18ms/step - loss: 0.8000 - rmse: 0.8181 - mse: 0.9974 - val_loss: 0.9184 - val_rmse: 0.9184 - val_mse: 0.8846 Epoch 29/98 57/57 [==============================] - 1s 19ms/step - loss: 0.7993 - rmse: 0.8170 - mse: 0.9961 - val_loss: 0.9183 - val_rmse: 0.9183 - val_mse: 0.8844 Epoch 30/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7990 - rmse: 0.8165 - mse: 0.9960 - val_loss: 0.9182 - val_rmse: 0.9182 - val_mse: 0.8842 Epoch 31/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7984 - rmse: 0.8155 - mse: 0.9952 - val_loss: 0.9181 - val_rmse: 0.9181 - val_mse: 0.8841 Epoch 32/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7979 - rmse: 0.8147 - mse: 0.9942 - val_loss: 0.9180 - val_rmse: 0.9180 - val_mse: 0.8839 Epoch 33/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7975 - rmse: 0.8139 - mse: 0.9938 - val_loss: 0.9180 - val_rmse: 0.9180 - val_mse: 0.8838 Epoch 34/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7970 - rmse: 0.8132 - mse: 0.9933 - val_loss: 0.9179 - val_rmse: 0.9179 - val_mse: 0.8837 Epoch 35/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7967 - rmse: 0.8124 - mse: 0.9925 - val_loss: 0.9178 - val_rmse: 0.9178 - val_mse: 0.8836 Epoch 36/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7962 - rmse: 0.8116 - mse: 0.9917 - val_loss: 0.9178 - val_rmse: 0.9178 - val_mse: 0.8835 Epoch 37/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7958 - rmse: 0.8109 - mse: 0.9914 - val_loss: 0.9177 - val_rmse: 0.9177 - val_mse: 0.8834 Epoch 38/98 57/57 [==============================] - 1s 19ms/step - loss: 0.7954 - rmse: 0.8102 - mse: 0.9905 - val_loss: 0.9177 - val_rmse: 0.9177 - val_mse: 0.8833 Epoch 39/98 57/57 [==============================] - 1s 19ms/step - loss: 0.7950 - rmse: 0.8095 - mse: 0.9901 - val_loss: 0.9177 - val_rmse: 0.9177 - val_mse: 0.8832 Epoch 40/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7947 - rmse: 0.8089 - mse: 0.9894 - val_loss: 0.9176 - val_rmse: 0.9176 - val_mse: 0.8832 Epoch 41/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7942 - rmse: 0.8081 - mse: 0.9887 - val_loss: 0.9176 - val_rmse: 0.9176 - val_mse: 0.8831 Epoch 42/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7939 - rmse: 0.8076 - mse: 0.9884 - val_loss: 0.9176 - val_rmse: 0.9176 - val_mse: 0.8831 Epoch 43/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7936 - rmse: 0.8069 - mse: 0.9877 - val_loss: 0.9175 - val_rmse: 0.9175 - val_mse: 0.8831 Epoch 44/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7930 - rmse: 0.8061 - mse: 0.9870 - val_loss: 0.9175 - val_rmse: 0.9175 - val_mse: 0.8830 Epoch 45/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7926 - rmse: 0.8054 - mse: 0.9865 - val_loss: 0.9175 - val_rmse: 0.9175 - val_mse: 0.8830 Epoch 46/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7922 - rmse: 0.8047 - mse: 0.9862 - val_loss: 0.9175 - val_rmse: 0.9175 - val_mse: 0.8829 Epoch 47/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7921 - rmse: 0.8044 - mse: 0.9860 - val_loss: 0.9175 - val_rmse: 0.9175 - val_mse: 0.8829 Epoch 48/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7917 - rmse: 0.8036 - mse: 0.9850 - val_loss: 0.9175 - val_rmse: 0.9175 - val_mse: 0.8829 Epoch 49/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7915 - rmse: 0.8032 - mse: 0.9849 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8829 Epoch 50/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7911 - rmse: 0.8025 - mse: 0.9843 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8829 Epoch 51/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7908 - rmse: 0.8019 - mse: 0.9839 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8828 Epoch 52/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7905 - rmse: 0.8014 - mse: 0.9832 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8828 Epoch 53/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7901 - rmse: 0.8007 - mse: 0.9825 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8828 Epoch 54/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7900 - rmse: 0.8004 - mse: 0.9827 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8828 Epoch 55/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7898 - rmse: 0.8000 - mse: 0.9824 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8828 Epoch 56/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7895 - rmse: 0.7994 - mse: 0.9819 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8828 Epoch 57/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7892 - rmse: 0.7989 - mse: 0.9815 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8827 Epoch 58/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7888 - rmse: 0.7983 - mse: 0.9806 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8827 Epoch 59/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7885 - rmse: 0.7977 - mse: 0.9805 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8827 Epoch 60/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7884 - rmse: 0.7974 - mse: 0.9804 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8827 Epoch 61/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7880 - rmse: 0.7968 - mse: 0.9797 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8827 Epoch 62/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7878 - rmse: 0.7964 - mse: 0.9795 - val_loss: 0.9174 - val_rmse: 0.9174 - val_mse: 0.8827 Epoch 63/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7876 - rmse: 0.7960 - mse: 0.9791 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 64/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7874 - rmse: 0.7956 - mse: 0.9787 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 65/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7872 - rmse: 0.7952 - mse: 0.9785 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 66/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7870 - rmse: 0.7948 - mse: 0.9783 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 67/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7867 - rmse: 0.7944 - mse: 0.9780 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 68/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7866 - rmse: 0.7940 - mse: 0.9777 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 69/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7865 - rmse: 0.7938 - mse: 0.9775 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 70/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7862 - rmse: 0.7933 - mse: 0.9770 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 71/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7861 - rmse: 0.7930 - mse: 0.9768 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 72/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7859 - rmse: 0.7926 - mse: 0.9765 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 73/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7856 - rmse: 0.7922 - mse: 0.9762 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 74/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7855 - rmse: 0.7920 - mse: 0.9760 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8827 Epoch 75/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7854 - rmse: 0.7916 - mse: 0.9760 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 76/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7850 - rmse: 0.7911 - mse: 0.9753 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 77/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7849 - rmse: 0.7909 - mse: 0.9754 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 78/98 57/57 [==============================] - 1s 16ms/step - loss: 0.7848 - rmse: 0.7906 - mse: 0.9752 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 79/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7846 - rmse: 0.7903 - mse: 0.9750 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 80/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7844 - rmse: 0.7900 - mse: 0.9748 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 81/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7842 - rmse: 0.7897 - mse: 0.9745 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 82/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7841 - rmse: 0.7895 - mse: 0.9744 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 83/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7839 - rmse: 0.7891 - mse: 0.9741 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 84/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7837 - rmse: 0.7888 - mse: 0.9739 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 85/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7835 - rmse: 0.7885 - mse: 0.9734 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 86/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7835 - rmse: 0.7884 - mse: 0.9736 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 87/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7834 - rmse: 0.7881 - mse: 0.9733 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 88/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7833 - rmse: 0.7879 - mse: 0.9732 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 89/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7830 - rmse: 0.7875 - mse: 0.9729 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 90/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7829 - rmse: 0.7873 - mse: 0.9729 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 91/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7827 - rmse: 0.7870 - mse: 0.9726 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 92/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7827 - rmse: 0.7869 - mse: 0.9728 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 93/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7824 - rmse: 0.7865 - mse: 0.9721 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 94/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7824 - rmse: 0.7865 - mse: 0.9723 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 95/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7822 - rmse: 0.7862 - mse: 0.9721 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 96/98 57/57 [==============================] - 1s 17ms/step - loss: 0.7821 - rmse: 0.7859 - mse: 0.9719 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 97/98 57/57 [==============================] - 1s 18ms/step - loss: 0.7820 - rmse: 0.7858 - mse: 0.9719 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826 Epoch 98/98 57/57 [==============================] - 1s 19ms/step - loss: 0.7818 - rmse: 0.7855 - mse: 0.9716 - val_loss: 0.9173 - val_rmse: 0.9173 - val_mse: 0.8826
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.8077 Root Mean Square Error: 1.0148 Mean Square Error: 1.0298
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.85 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (348, 432, 1) 11/11 [==============================] - 0s 4ms/step - loss: 0.7280 - rmse: 0.7280 - mse: 0.9919 evaluate: [0.7280444502830505, 0.728044331073761, 0.9918908476829529]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
LSTM: Mean Absolute Error: 0.6755 Root Mean Square Error: 0.9095 Mean Square Error: 0.8273
anomalies: (180, 10)
###################################################### ####################### PM25 ########################### Testing shape: (348, 432, 1) 11/11 [==============================] - 0s 4ms/step - loss: 0.7668 - rmse: 0.7668 - mse: 1.1052 evaluate: [0.7667776346206665, 0.7667775750160217, 1.1051918268203735]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.7076 Root Mean Square Error: 0.9296 Mean Square Error: 0.8642
anomalies: (7, 10)
###################################################### ####################### PM10 ########################### Testing shape: (348, 432, 1) 11/11 [==============================] - 0s 4ms/step - loss: 0.7975 - rmse: 0.7975 - mse: 1.1793 evaluate: [0.7974935173988342, 0.7974935173988342, 1.1792500019073486]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.7341 Root Mean Square Error: 0.9429 Mean Square Error: 0.8890
anomalies: (1, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (348, 432, 1) 11/11 [==============================] - 0s 4ms/step - loss: 0.7974 - rmse: 0.7974 - mse: 1.0300 evaluate: [0.7973957061767578, 0.7973962426185608, 1.0299934148788452]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.7431 Root Mean Square Error: 0.9346 Mean Square Error: 0.8735
anomalies: (0, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (348, 432, 1) 11/11 [==============================] - 0s 4ms/step - loss: 0.7859 - rmse: 0.7859 - mse: 1.0147 evaluate: [0.7859255075454712, 0.7859252691268921, 1.0147411823272705] LSTM:
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
Mean Absolute Error: 0.7333 Root Mean Square Error: 0.9269 Mean Square Error: 0.8591
anomalies: (0, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (348, 432, 1) 11/11 [==============================] - 0s 4ms/step - loss: 0.7765 - rmse: 0.7765 - mse: 1.0278 evaluate: [0.7764676809310913, 0.7764678597450256, 1.027838110923767]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.7231 Root Mean Square Error: 0.9269 Mean Square Error: 0.8592
anomalies: (0, 10)
######################################################